ChessVision Performance Report¶
This notebook generates a full performance report of the ChessVision system.
Imports¶
In [1]:
%matplotlib inline
from mpl_toolkits.axes_grid1 import ImageGrid
import matplotlib.pyplot as plt
import chessvision as cv
from test import run_tests, get_test_generator, plot_confusion_mtx, labels
import cv_globals
import cv2
import numpy as np
from model.u_net import load_extractor
from model.square_classifier import load_classifier
Parameters¶
These variables will be filled in by papermill. (Default to using current best models)
In [2]:
classifier_weights = cv_globals.square_weights
extractor_weights = cv_globals.board_weights
threshold = 80
In [3]:
# Parameters
extractor_weights = "C:\\Users\\Gudbrand\\Programming\\ChessVision\\weights\\best_extractor.hdf5"
classifier_weights = "C:\\Users\\Gudbrand\\Programming\\ChessVision\\weights\\best_classifier.hdf5"
threshold = 80
Load models and test data generator¶
In [4]:
data_generator = get_test_generator()
extractor = load_extractor(weights=extractor_weights)
classifier = load_classifier(weights=classifier_weights)
In [5]:
results = run_tests(data_generator, extractor, classifier, threshold)
Get results¶
In [6]:
board_imgs = results["board_imgs"]
imgs = results["raw_imgs"]
predictions = results["predictions"]
boards = results["chessboards"]
squares = results["squares"]
acc = results["acc"]
board_accs = results["board_accs"]
avg_time = results["avg_time"]
avg_entropy = results["avg_entropy"]
squares = results["squares"]
masks = results["masks"]
errors = results["errors"]
top_2 = results["top_2_accuracy"]
top_3 = results["top_3_accuracy"]
ondiag, offdiag = results["hits"]
N = len(results["raw_imgs"])
In [7]:
print("Test suite accuracy: {:.1f}%".format(acc*100))
print("Top-2 accuracy: {:.1f}%".format(top_2*100))
print("Top-3 accuracy: {:.1f}%".format(top_3*100))
print("Average time per raw classification: {:.1f}s".format(avg_time))
print("Average entropy of classifier predictions: {:.2g}".format(avg_entropy))
print("Number of board extraction errors: {}".format(errors))
print("Overall correct piece classifications: {}/{} ({} incorrect)".format(ondiag, ondiag + offdiag, offdiag))
Confusion Matrix¶
In [8]:
mtx = results["confusion_matrix"]
plt.figure(figsize=(9, 9))
plot_confusion_mtx(mtx, labels)
plt.show()
Plot results¶
In [9]:
rows = N
cols = 3
fig = plt.figure(1, figsize=(12, 40)) #width, height in inches.
grid = ImageGrid(fig, 111, # similar to subplot(111)
nrows_ncols=(rows, cols), # creates 2x2 grid of axes
axes_pad=0.1, # pad between axes in inch.
share_all=True,
label_mode=None
)
for i in range(rows):
raw = imgs[i]
board = board_imgs[i]
mask = cv2.resize(masks[i], (512, 512))
ax1 = grid[cols*i]
ax1.imshow(raw)
ax1.axis("off")
ax2 = grid[cols*i+1]
ax2.imshow(mask.reshape((512, 512)), cmap="gray")
ax2.axis("off")
ax3 = grid[cols*i+2]
ax3.imshow(board, cmap="gray")
ax3.axis("off")
if i == 0:
ax1.set_title("Raw")
ax2.set_title("Predicted mask")
ax3.set_title("Extracted board")
#plt.savefig("../../img/training_extraction.png", bbox_inches="tight")
plt.show()
In [10]:
label_names = ['B', 'K', 'N', 'P', 'Q', 'R', 'b', 'k', 'n', 'p', 'q', 'r', 'f']
for k in range(N):
prediction = np.argmax(predictions[k], axis=1)
pred_labels = [label_names[p] for p in prediction]
square_grid = squares[k]
# Plot
rows, cols = 8, 8
fig = plt.figure(figsize=(12, 12))
grid = ImageGrid(fig, 111, # similar to subplot(111)
nrows_ncols=(rows, cols), # creates 2x2 grid of axes
axes_pad=0.3, # pad between axes in inch.
share_all=True,
label_mode=None
)
for i in range(cols):
for j in range(rows):
ind = cols*i+j
im = square_grid[ind]
label = pred_labels[ind]
grid[ind].imshow(im.reshape(64, 64), cmap="gray")
grid[ind].set_title(label, size=14, fontweight=3)
grid[ind].axis("off")
plt.suptitle("Classification of test image {}: {}/64".format(k, int(64*board_accs[k])), size=20, fontweight=5)
In [ ]: